#include <iostream>
#include <fstream>
#include <string>
#include <vector>
#include <cstdio>
#include <sstream>
#include <iomanip>
#include <windows.h>
#include <utility>

#include "utils.h"
#include "logger.h"
#include "helper.h"
#include "block.h"
#include "domain.h"

using namespace std;

//Init function {
bool Startup();
void Cleanup();
void MainService();
void BreakProcess();
//}

//Utility function {
bool ParseConfigFile(const char * pszFileName);
vector<int> ParseFileName(const string & filename);
bool DeleteFiles(int key);
void BuildQueryData(vector<Record> * records);
bool LoadSearchResults(vector<vector<vector<unsigned int> > > * results);
bool LoadImgVotes(vector<map<string, map<string, 
				  pair<unsigned int, unsigned int> > > > * imgvotes);
//}

//Serial function {
template<class T>
bool LoadDataBlock(DataBlock<T> * db, const string & filename);
template<class T>
bool SaveDataBlock(DataBlock<T> * db, const string & filename);
//}

//Hash function {
void HashProcess();
bool ParseDataSource();
bool GetSubVectors();
bool ComputeL2Norm();
bool FindMedian();
bool HashMapping();
//}

//Search function {
void SearchProcess();
bool MatchPoint();
bool MatchImage();
bool MatchVideo();
//}


//Global Variables {
SyncLogger * pInfoLog;
Config * pConf;
BreakLogger * pBreakLog;
char szRawDataFile[512];
char szQueryFile[512];
string workDir;
bool isRunning;
int stopPoint = -1;
unsigned int totalRecord = 0;

const char * PRO_TIPS_ERR[] = {
	"Error: Parse data source for hashing failed!",
	"Error: Get sub vectors failed!",
	"Error: Compute L2 norm for each sub vector failed!",
	"Error: Find median value for L2 norm failed!",
	"Error: Mapping failed!",
	"Error: Parse query data source for searching failed!",
	"Error: Point matching failed!",
	"Error: Image matching failed!",
	"Error: Video matching failed!"
};

const char * PRO_TIPS_BEGIN[] = {
	"Info: Parse from data source.",
	"Info: Generate sub vectors.",
	"Info: Compute L2 norms.",
	"Info: Find median values.",
	"Info: Hash Mapping.",
	"Info: Parse query data.",
	"Info: Point matching.",
	"Info: Image matching.",
	"Info: Video matching."
};

const char * PRO_TIPS_DONE[] = {
	"Info: Parse data source done.",
	"Info: Get sub vectors done.",
	"Info: Compute L2 norm done.",
	"Info: Find median value for L2 norm done.",
	"Info: Mapping done.",
	"Info: Parse query data done.",
	"Info: Point matching done.",
	"Info: Image matching done.",
	"Info: Video matching done."
};
//}

int main(int argc, char * argv[]) {
	if (!Startup()) {
		pInfoLog->Log("Error: An error occurred while starting.");
	}
	MainService();
	Cleanup();
	return 0;
}

bool Startup() {
	char szLogFile[512], szConfFile[512], szBreakFile[512];

	// Construct log and config filenames
	GetModuleFileName(0,szLogFile,512);
	*strrchr(szLogFile, '\\') = 0;
	workDir = szLogFile;
	workDir += + "\\work";
	strcpy_s(szConfFile,szLogFile);
	strcpy_s(szBreakFile, szConfFile);
	strcpy_s(szRawDataFile, szBreakFile);
	strcpy_s(szQueryFile, szRawDataFile);

	strcat_s(szLogFile, "\\hsnns.log");
	strcat_s(szConfFile, "\\hsnns.conf");
	strcat_s(szBreakFile, "\\hsnns.break");
	strcat_s(szRawDataFile, "\\raw\\test.txt");
	strcat_s(szQueryFile, "\\raw\\queryVideoDescriptors.txt");

	// Start logger thread
	pInfoLog = new SyncLogger(szLogFile, true);
	pBreakLog = new BreakLogger(szBreakFile);
	pConf = new Config();
	if (!ParseConfigFile(szConfFile)) return false;
	stopPoint = pConf->GetVal(RUN_STOP_POINT);
	// Log some startup info
	if (pBreakLog->info->processId > pConf->GetVal(RUN_STOP_POINT)) {
		pInfoLog->Log("Error: Check stop point setting.");
		exit(0);
	}
	pInfoLog->Log("-------------------------------------------------------------------------------");
	pInfoLog->Log("Info: HSNNS is startup.");
	return true;
}

void Cleanup() {
	pInfoLog->Log("Info: Done.");
	delete pInfoLog;
	delete pConf;
}

bool ParseConfigFile(const char * pszFileName) {
	ifstream fin(pszFileName, ifstream::in);
	string line;

	if (!fin.good()) {
		pInfoLog->Log("Error: Unable to open \"hsnns.conf\".");
		return false;
	}

	while (getline(fin, line, '\n')) {
		if (line.find_first_of('#') != 0 && !line.empty()) {
			vector<string> prop = split(line, "=");
			pConf->SetVal(trim(prop[0]), atoi(trim(prop[1]).c_str()));
		}
	}

	return true;
}

void MainService() {
	isRunning = true;
	while(isRunning) {
		unsigned int pid = pBreakLog->info->processId;
		pBreakLog->info->isHash = (pid > H_IND_HASHMAP)? false : true;
		if (pBreakLog->info->isHash) {
			if (pid < H_RAW_DATA || pid > H_IND_HASHMAP) {
				pBreakLog->SetNewState(H_RAW_DATA);
			}
			HashProcess();
		} else {
			if (pid < S_RAW_DATA || pid > S_MATCH_VID) {
				pBreakLog->SetNewState(S_RAW_DATA);
			}
			SearchProcess();
		}
	}
}

void HashProcess() {
	bool (*func) ();
	unsigned int pid = pBreakLog->info->processId;
	switch(pid) {
		case H_RAW_DATA :
			func = ParseDataSource; break;
		case H_IND_SUB_VECTOR : 
			func = GetSubVectors; break;
		case H_IND_L2NORM : 
			func = ComputeL2Norm; break;
		case H_IND_MEDIAN : 
			func = FindMedian; break;
		case H_IND_HASHMAP : 
			func = HashMapping; break;
	}
	
	pInfoLog->Log(PRO_TIPS_BEGIN[pid]);
	if (!(*func)()) {
		pInfoLog->Log(PRO_TIPS_ERR[pid]);
	} else {
		pInfoLog->Log(PRO_TIPS_DONE[pid]);
	}
	isRunning = (stopPoint != pid) ? true : false;
	return;
}

void SearchProcess() {
	bool (*func) ();
	unsigned int pid = pBreakLog->info->processId;
	switch(pid) {
		case S_RAW_DATA :
			func = ParseDataSource; break;
		case S_MATCH_PNT : 
			func = MatchPoint; break;
		case S_MATCH_IMG : 
			func = MatchImage; break;
		case S_MATCH_VID :
			func = MatchVideo; break;
	}

	pInfoLog->Log(PRO_TIPS_BEGIN[pid]);
	if (!(*func)()) {
		pInfoLog->Log(PRO_TIPS_ERR[pid]);
	} else {
		pInfoLog->Log(PRO_TIPS_DONE[pid]);
	}
	isRunning = (stopPoint != pid) ? true : false;
	return;
}

bool ParseDataSource() {
	char * szfile = szRawDataFile;
	//raw_[block index]_[start index]_[block size].dat
	char * filefmt = "raw_%d_%d.dat";
	int key = H_RAW_DATA;
	int helperKey = H_RAW_IMG_HELPER;
	int nextKey = H_IND_SUB_VECTOR;
	string helper = "raw_img_helper.dat";
	if (!pBreakLog->info->isHash) {
		szfile = szQueryFile;
		filefmt = "qraw_%d_%d.dat";
		key = S_RAW_DATA;
		helper = "query_img_helper.dat";
		nextKey = S_MATCH_PNT;
		helperKey = S_QUERY_IMG_HELPER;
	}

	if (NULL == szfile) {
		pInfoLog->Log("Error: Bad Raw/Query data file name. (null)");
		return false;
	}

	ifstream fin(szfile, ifstream::in);

	if (!fin.good()) {
		pInfoLog->Log("Error: Unable to open Raw/Query data file. (%s)", szfile);
		return false;
	}
	unsigned int maxFileNodeCount = pConf->GetVal(CFG_MAX_RAW_DATA_BS);
	FileHelper *imgFileHelper = new FileHelper();
	unsigned int doneCount = 0;
	unsigned int totalCount = 0;
	if (pBreakLog->HasBreakPoint()) {
		unsigned int fileId = pBreakLog->info->fileInfo[key].size() - 1;
		fileId = fileId < 0 ? 0 : fileId;
		totalCount = (pBreakLog->info->blockId - 1) * maxFileNodeCount;
		doneCount = totalCount;
		LoadFileHelper(imgFileHelper, workDir + "\\" + helper);
		pInfoLog->Log("Info: Restart from breakpoint.(%s)", 
			pBreakLog->info->fileInfo[key][fileId].c_str());
	} else {
		pBreakLog->SetToDoFlag();
	}
	DataBlock<Record> * db = new DataBlock<Record>(totalCount, maxFileNodeCount);
	string line;
	while (getline(fin, line, '\n')) {
		if (doneCount > 0) {
			doneCount --;
			continue;
		}

		Record record(line);
		db->Add(record);
		imgFileHelper->AddFileName(record.imageFileName, totalCount);
		totalCount ++;
		if (totalCount % maxFileNodeCount == 0) {
			SaveFileHelper(imgFileHelper, workDir + "\\" + helper);
			pBreakLog->UpdateBreakInfo(helperKey, helper, false);
			string filename = db->GetFileName(filefmt, db->GetStartIndex(), db->GetCount());
			SaveDataBlock(db, filename);
			pBreakLog->UpdateBreakInfo(key, filename, true);
			
			delete db;
			db = NULL;
			db = new DataBlock<Record>(totalCount, maxFileNodeCount);
		}
		
	}
	fin.close();
	imgFileHelper->BuildPosition();
	SaveFileHelper(imgFileHelper, workDir + "\\" + helper);
	pBreakLog->UpdateBreakInfo(helperKey, helper, false);
	string filename = db->GetFileName(filefmt, db->GetStartIndex(), db->GetCount());
	SaveDataBlock(db, filename);
	pBreakLog->UpdateBreakInfo(key, filename, true);
	delete db;
	db = NULL;
	delete imgFileHelper;
	imgFileHelper = NULL;
	pBreakLog->SetNewState(nextKey);
	return true;
}

bool GetSubVectors() {
	//sub_[rebuild index]_[sub vector num].dat
	char * filefmt = "sub_%d_%d.dat";
	unsigned int dim = pConf->GetVal(CFG_SUB_VEC_DIM);
	unsigned int num = pConf->GetVal(CFG_SUB_VEC_NUM);
	unsigned int rcdDim = pConf->GetVal(CFG_RECORD_DIM);
	unsigned int reCount = pConf->GetVal(CFG_REBUILD_COUNT);

	unsigned int startIndex = 0;
	if (pBreakLog->HasBreakPoint()) {
		unsigned int fileId = pBreakLog->info->fileInfo[H_IND_SUB_VECTOR].size() - 1;
		fileId = fileId < 0 ? 0 : fileId;
		startIndex = pBreakLog->info->blockId - 1;
		pInfoLog->Log("Info: Restart from breakpoint.(%s)", 
			pBreakLog->info->fileInfo[H_IND_SUB_VECTOR][fileId].c_str());
	} else {
		pBreakLog->SetToDoFlag();
	}

	for (unsigned int i = startIndex; i < reCount; i++) {
		unsigned int n = rcdDim - dim;
		vector<int> tmp(n); 
		for (unsigned int j = 0; j < n; j ++) {
			tmp[j] = j;
		}
		random_shuffle(tmp.begin(), tmp.end());

		DataBlock<SubVector> *db = new DataBlock<SubVector>(i, num);
		for (unsigned int j = 0; j < num; j ++) {
			SubVector sv(tmp[j], dim);
			db->Add(sv);
		}
		string filename = db->GetFileName(filefmt, db->GetStartIndex(), num);
		SaveDataBlock(db, filename);
		pBreakLog->UpdateBreakInfo(H_IND_SUB_VECTOR, filename, true);
		delete db;
		db = NULL;
	}

	pBreakLog->SetNewState(H_IND_L2NORM);
	return true;
}

bool ComputeL2Norm() {
	//l2n_[rebuild index]_[raw data block index].dat
	char * filefmt = "l2n_%d_%d.dat";
	vector<string> subfiles = pBreakLog->info->fileInfo[H_IND_SUB_VECTOR];
	vector<string> rawfiles = pBreakLog->info->fileInfo[H_RAW_DATA];
	unsigned int reCount = pConf->GetVal(CFG_REBUILD_COUNT);
	unsigned int subvecnum = pConf->GetVal(CFG_SUB_VEC_NUM);
	vector<vector<SubVector> > subvectors(reCount);
	for (unsigned int j = 0; j < subfiles.size(); j ++) {
		DataBlock<SubVector> * subdb = new DataBlock<SubVector>();
		LoadDataBlock(subdb, subfiles[j]);
		subvectors[j] = subdb->dataSet;
		delete subdb;
		subdb = NULL;
	}

	unsigned int rebuildIndex = 0;
	unsigned int blockIndex = 0;
	bool isUsed = true;
	if (pBreakLog->HasBreakPoint()) {
		unsigned int l2nsize = pBreakLog->info->fileInfo[H_IND_L2NORM].size();
		unsigned int rawsize = rawfiles.size();
		unsigned int fileId = l2nsize - 1;
		fileId = fileId < 0 ? 0 : fileId;
		rebuildIndex = l2nsize / rawsize;
		blockIndex = l2nsize % rawsize;
		pInfoLog->Log("Info: Restart from breakpoint.(%s)", 
			pBreakLog->info->fileInfo[H_IND_L2NORM][fileId].c_str());
		isUsed = false;
	} else {
		pBreakLog->SetToDoFlag();
	}

	for (unsigned int k = rebuildIndex; k < reCount; k++) {
		if (isUsed) {
			blockIndex = 0;
		} else {
			isUsed = true;
		}
		unsigned int rawfileNum = rawfiles.size();
		for (unsigned int i = blockIndex; i < rawfileNum; i++) {
			DataBlock<Record> * rawdb = new DataBlock<Record>();
			LoadDataBlock(rawdb, rawfiles[i]);
			DataBlock<L2Norms> * l2ndb = new DataBlock<L2Norms>(0, subvecnum);

			for (unsigned int j = 0; j < subvectors[k].size(); j++) {
				L2Norms l2norms;
				l2norms.l2norm.resize(rawdb->dataSet.size());
				for (unsigned int p = 0; p < rawdb->dataSet.size(); p++) {
					unsigned char s = subvectors[k][j].startIndex;
					unsigned char d = subvectors[k][j].subVectorDim;

					double sum = 0.0;
					for (unsigned char q = s; q < s + d; q ++) {
						sum += rawdb->dataSet[p].eigenVector[q] ^ 2;
					}
					double l2 = sqrt(sum);
					l2norms.l2norm[p] = l2;
				}
				l2ndb->Add(l2norms);
			}

			string filename = l2ndb->GetFileName(filefmt, k, i);
			SaveDataBlock(l2ndb, filename);
			pBreakLog->UpdateBreakInfo(H_IND_L2NORM, filename, true);
			delete l2ndb;
			l2ndb = NULL;

			delete rawdb;
			rawdb = NULL;
		}
	}
	
	pBreakLog->SetNewState(H_IND_MEDIAN);
	return true;
}

bool FindMedian() {
	unsigned int reCount = pConf->GetVal(CFG_REBUILD_COUNT);
	unsigned int nodeSize = pConf->GetVal(CFG_MAX_RAW_DATA_BS);
	unsigned int subNum = pConf->GetVal(CFG_SUB_VEC_NUM);
	unsigned int rawSize = pBreakLog->info->fileInfo[H_RAW_DATA].size();
	vector<string> l2nfiles = pBreakLog->info->fileInfo[H_IND_L2NORM]; 
	DataBlock<Record> * rawdb = new DataBlock<Record>();
	LoadDataBlock(rawdb, pBreakLog->info->fileInfo[H_RAW_DATA][rawSize - 1]);
	unsigned int total = (rawSize - 1) * nodeSize + rawdb->dataSet.size();
	unsigned int medId = total / 2;

	//rebuildId, subVectorId, l2norm
	vector<vector<map<unsigned int, unsigned int> > > hist(reCount);
	for (unsigned int i = 0; i < hist.size(); i++) {
		vector<map<unsigned int, unsigned int> > subvecId(subNum);
		for (unsigned int j = 0; j < subvecId.size(); j++) {
			map<unsigned int, unsigned int> tmp;
			subvecId[j] = tmp;
		}
		hist[i] = subvecId;
	}

	for (unsigned i = 0; i < reCount; i ++) {
		for (unsigned j = 0; j < rawSize; j ++) {
			DataBlock<L2Norms> * l2ndb = new DataBlock<L2Norms>();
			LoadDataBlock(l2ndb, l2nfiles[i * rawSize + j]);
			for (unsigned int k = 0; k < subNum; k ++) {
				for(unsigned int p = 0; p < l2ndb->dataSet[k].l2norm.size(); p++) {
					unsigned int l2n = static_cast<unsigned int>(l2ndb->dataSet[k].l2norm[p] + 0.5f);
					if (hist[i][k].find(l2n) == hist[i][k].end()) {
						hist[i][k][l2n] = 1;
					} else {
						hist[i][k][l2n] += 1;
					}
				}
			}
			delete l2ndb;
			l2ndb = NULL;
		}
	}

	for (unsigned int i = 0; i < reCount; i ++) {
		DataBlock<double> * meddb = new DataBlock<double>(0, subNum);
		for (unsigned int j = 0; j < subNum; j ++) {
			map<unsigned int , unsigned int>::iterator it;
			unsigned int count = 0;
			unsigned int med = 0;
			for (it = hist[i][j].begin(); it != hist[i][j].end(); it++) {
				count += it->second;
				if (count > medId) {
					med = it->first;
					break;
				}
			}
			meddb->Add(med);
		}
		string filename = meddb->GetFileName("med_%d.dat", i);
		SaveDataBlock(meddb, filename);
		pBreakLog->UpdateBreakInfo(H_IND_MEDIAN, filename, false);
		delete meddb;
		meddb = NULL;
	}

	pBreakLog->SetNewState(H_IND_HASHMAP);
	return true;
}

bool HashMapping() {
	unsigned int reCount = pConf->GetVal(CFG_REBUILD_COUNT);
	unsigned int rawSize = pBreakLog->info->fileInfo[H_RAW_DATA].size();
	vector<string> medfiles = pBreakLog->info->fileInfo[H_IND_MEDIAN];
	vector<string> l2nfiles = pBreakLog->info->fileInfo[H_IND_L2NORM];
	DataBlock<double> * meddb = NULL;
	DataBlock<Index> * inddb = NULL;

	unsigned int rebuildId = 0;
	if (pBreakLog->HasBreakPoint()) {
		rebuildId = pBreakLog->info->fileInfo[H_IND_HASHMAP].size() - 1;
		rebuildId = rebuildId < 0 ? 0 : rebuildId;
		pInfoLog->Log("Info: Restart from breakpoint.(%s)", pBreakLog->info->fileInfo[H_IND_HASHMAP][rebuildId]);
	} else {
		pBreakLog->SetToDoFlag();
	}

	for (unsigned int i = rebuildId; i < reCount; i ++) {
		Index index;
		inddb = new DataBlock<Index>(0, 1);
		meddb = new DataBlock<double>();
		LoadDataBlock(meddb, medfiles[i]);
		vector<double> medians = meddb->dataSet;
		delete meddb;
		meddb = NULL;

		DataBlock<L2Norms> * l2ndb = NULL;
		vector<unsigned int> hashval;
		unsigned int rstart = 0;
		unsigned int rcount = 0;

		for (unsigned int j = 0; j < rawSize; j++) {
			l2ndb = new DataBlock<L2Norms>();
			LoadDataBlock(l2ndb, l2nfiles[j + i * rawSize]);

			unsigned int svnum = l2ndb->dataSet.size(); // sub vector num
			unsigned int bsize = l2ndb->dataSet[0].l2norm.size(); //block size;
			rcount = bsize;
			hashval.resize(hashval.size() + bsize);

			for (unsigned int k = 0; k < rcount; k ++) {
				hashval[k + rstart] = 0;
			}
			for (unsigned int k = 0; k < svnum; k ++) {
				for (unsigned int l = 0; l < bsize; l++) {
					hashval[l + rstart] <<= 1;
					if(l2ndb->dataSet[k].l2norm[l] < medians[k]) {
						hashval[l + rstart] += 0;
					} else {
						hashval[l + rstart] += 1;
					}
				}
			}

			for (unsigned int k = 0; k < rcount; k++) {
				index.indexs[hashval[k + rstart]].push_back(k + rstart);
			}

			delete l2ndb;
			l2ndb = NULL;
			rstart += rcount;
		}
		inddb->Add(index);
		string filename = inddb->GetFileName("ind_%d.dat", i);
		SaveDataBlock(inddb, filename);
		pBreakLog->UpdateBreakInfo(H_IND_HASHMAP, filename, true);
		delete inddb;
		inddb = NULL;
	}

	DeleteFiles(H_IND_L2NORM);
	pBreakLog->info->isHash = false;
	pBreakLog->SetToDoFlag();
	pBreakLog->SetNewState(S_RAW_DATA);
	return true;
}

bool MatchPoint() {
	unsigned int reCount = pConf->GetVal(CFG_REBUILD_COUNT);
	unsigned int subvecnum = pConf->GetVal(CFG_SUB_VEC_NUM);
	vector<Record> * records = new vector<Record>();
	BuildQueryData(records);

	pInfoLog->Log("Info: Compute l2 norm.");
	vector<string> subfiles = pBreakLog->info->fileInfo[H_IND_SUB_VECTOR];
	unsigned int recsize = records->size();
	unsigned int subfilesize = subfiles.size();
	vector<vector<vector<double> > > * ql2n = new vector<vector<vector<double> > >(recsize);

	for (unsigned int i = 0; i < recsize; i++) {
		vector<vector<double> > tmp(subfilesize);
		for (unsigned int j = 0; j < subfilesize; j ++) {
			vector<double> tmp2(subvecnum);
			tmp[j] = tmp2;
		}
		ql2n->at(i) = tmp;
	}

	DataBlock<SubVector> * subdb = NULL;
	for (unsigned int i = 0; i < subfiles.size(); i++) {
		subdb = new DataBlock<SubVector>();
		LoadDataBlock(subdb, subfiles[i]);

		for (unsigned int j = 0; j < subdb->dataSet.size(); j++) {
			unsigned int s = subdb->dataSet[j].startIndex;
			unsigned int l = subdb->dataSet[j].subVectorDim;

			for (unsigned int p = 0; p < recsize; p ++) {
				double sum = 0.0;
				for (unsigned int k = s; k < s + l; k ++) {
					sum += records->at(p).eigenVector[k] ^ 2;
				}
				(*ql2n)[p][i][j] = sqrt(sum);
			}
		}
		delete subdb;
		subdb = NULL;
	}

	pInfoLog->Log("Info: Hash Mapping.");
	vector<string> medfiles = pBreakLog->info->fileInfo[H_IND_MEDIAN];
	unsigned int medfilesize = medfiles.size();
	vector<vector<unsigned int> > hasVals(recsize);
	for (unsigned int i = 0; i < recsize; i++) {
		vector<unsigned int> tmp(medfilesize, 0);
		hasVals[i] = tmp;
	}
	DataBlock<double> *meddb = NULL;
	for (unsigned int i = 0; i < medfiles.size(); i ++) {
		meddb  = new DataBlock<double>();
		LoadDataBlock(meddb, medfiles[i]);
		for (unsigned int j = 0; j < meddb->dataSet.size(); j ++) {
			for (unsigned int p = 0; p < recsize; p ++) {
				hasVals[p][i] <<= 1;
				if ((*ql2n)[p][i][j] < meddb->dataSet[j]) {
					hasVals[p][i] += 0;
				} else {
					hasVals[p][i] += 1;
				}
			}
		}
		delete meddb;
		meddb;
	}

	
	delete ql2n;
	ql2n = NULL;

	pInfoLog->Log("Info: Searching.");
	vector<string> indfiles = pBreakLog->info->fileInfo[H_IND_HASHMAP];
	vector<vector<vector<unsigned int> > > * results = new vector<vector<vector<unsigned int> > >(recsize);
	for (unsigned int i = 0; i < recsize; i ++) {
		vector<vector<unsigned int> > tmp(reCount);
		results->at(i) = tmp;
	}
	DataBlock<Index> * inddb = NULL;
	for (unsigned int i = 0; i < indfiles.size(); i++) {
		pInfoLog->Log("Info: Seeking index : %s", indfiles[i].c_str());
		inddb = new DataBlock<Index>();
		LoadDataBlock(inddb, indfiles[i]);

		for (unsigned int p = 0; p < recsize; p ++) {
			unsigned int key = hasVals[p][i];
			map<unsigned int, vector<unsigned int> >::iterator it;
			if ((it = inddb->dataSet[0].indexs.find(key)) != inddb->dataSet[0].indexs.end()) {
				for (unsigned int j = 0; j < it->second.size(); j++) {
					(*results)[p][i].push_back((it->second)[j]);
				}
			}
		}

		delete inddb;
		inddb = NULL;
	}

	if (pConf->GetVal(RUN_MATCH_POINT) > 0) {
		string readname = "match_points.txt";
		string tmpname = workDir + "\\" + readname;
		ofstream fout(tmpname.c_str());
		for (unsigned int i = 0, n = results->size(); i < n; i++) {
			fout << i << " : ";
			for(unsigned int j = 0, m = results->at(i).size(); j < m; j++) {
				for (unsigned int k = 0, q = (*results)[i][j].size(); k < q; k++) {
					fout << (*results)[i][j][k] << " ";
				}
				fout << " | ";
			}
			fout << endl;
		}
		fout.close();
		pBreakLog->UpdateBreakInfo(S_READABLE_RES, readname, false);
	}

	string filename = "match_points.dat";
	string output = workDir + "\\" + filename;
	ofstream ofs(output.c_str(), ios::binary);
	boost::archive::binary_oarchive oa(ofs);
	oa << *results;
	ofs.close();
	pBreakLog->UpdateBreakInfo(S_MATCH_PNT, filename, false);
	pInfoLog->Log("Info: Save match result to (%s).", filename.c_str());
	delete records;
	delete results;
	pBreakLog->SetNewState(S_MATCH_IMG);
	return true;
}

bool MatchImage() {
	pInfoLog->Log("Info: Prepare data.");
	unsigned int reCount = pConf->GetVal(CFG_REBUILD_COUNT);
	vector<Record> * records = new vector<Record>();
	BuildQueryData(records);
	vector<vector<vector<unsigned int> > > * results = new vector<vector<vector<unsigned int> > >();
	if(!LoadSearchResults(results)) {
		pInfoLog->Log("Error: Cannot load search result from point matching.");
		return false;
	}

	//mission: picture matching
	//0.hold all query record.
	//for each query picture: 
	//1.build votes data struct.
	//2.get all record from library.
	//3.1.compute Euclidean distance.
	//3.2.compute score for each picture.
	//4.find matching picture by threshold.

	//get ImageFileHelper
	string imghelpername = pBreakLog->info->fileInfo[H_RAW_IMG_HELPER][0];
	string qimghelpername = pBreakLog->info->fileInfo[S_QUERY_IMG_HELPER][0];
	FileHelper * img = new FileHelper();
	FileHelper * queryimg = new FileHelper();
	LoadFileHelper(img, workDir + "\\" + imghelpername);
	LoadFileHelper(queryimg, workDir + "\\" + qimghelpername);
	unsigned int voteSize = queryimg->indexInfo.size();
	unsigned int voteI = 0;
	unsigned int recsize = records->size();

	//save result
	vector<pair<string, pair<string, unsigned int> > > match;
	//save all votes, string=image filename, unsigned int=score;
	vector<map<string, map<string, pair<unsigned int, unsigned int> > > > votes(voteSize);
	//save distance, unsigned int=match vector index, double=o-distance
	vector<map<unsigned int, double> > distans(recsize);

	//save angles, string=angle key, vector=vector indexes
	vector<map<string, vector<unsigned int> > > angles(recsize);

	//save index info for 2.
	map<unsigned int, pair<string, vector<unsigned char> > > indexVector;
	map<unsigned int, pair<string, vector<unsigned char> > >::iterator iit;

	//1.begin:
	pInfoLog->Log("Info: Build votes data struct.");
	map<string, pair<unsigned int, unsigned int> >::iterator it;
	for (it = queryimg->fileInfo.begin(); it != queryimg->fileInfo.end(); it++) {
		map<string, map<string, pair<unsigned int, unsigned int> > > vote;
		for (unsigned int i = it->second.first, n = it->second.second + it->second.first; i < n; i++) {
			map<unsigned int, double> tmp;
			map<string, vector<unsigned int> > tmp3;
			for(unsigned int j = 0, m = (*results)[i].size(); j < m; j++) {
				for (unsigned k = 0, l = (*results)[i][j].size(); k < l; k++) {
					unsigned int index = (*results)[i][j][k];
					string key = img->GetFileName(index);
					map<string, pair<unsigned int, unsigned int> > tmp4;
					vote[key] = tmp4;
					tmp[index] = 0.0;
					pair<string, vector<unsigned char> > tmp2;
					indexVector[index] = tmp2;
				}
			}
			distans[i] = tmp;
			angles[i] = tmp3;
		}
		votes[voteI++] = vote;
	}

	delete results;
	results = NULL;

	//2.begin:
	pInfoLog->Log("Info: Get all records from txt.");
	ifstream fin(szRawDataFile);
	string line;
	unsigned int linecount = 0;
	iit = indexVector.begin();

	while(getline(fin, line, '\n') && iit != indexVector.end()) {
		if (linecount == iit->first) {
			Record record(line);
			iit->second.second = record.eigenVector;
			ostringstream ostr;
			ostr << setiosflags(ios::fixed) << setprecision(6);
			ostr << record.metadatas[0] << '|' << record.metadatas[1];
			iit->second.first = ostr.str();
			iit ++;
		}
		linecount ++;
	}
	fin.close();


	//0.begin:

	//3.begin:
	pInfoLog->Log("Info: Compute Euclidean distance and score.");
	map<string, double>::iterator vit;
	unsigned int voteIndex = 0;
	for (it = queryimg->fileInfo.begin(); it != queryimg->fileInfo.end(); it++) {
		for (unsigned int i = it->second.first, n = it->second.second + it->second.first; i < n; i++) {

			map<unsigned int, double>::iterator dit;
			for (dit = distans[i].begin(); dit != distans[i].end(); dit++) {
				double sum = 0.0;
				for (unsigned int j = 0; j < 128; j ++) {
					unsigned int tmp = (indexVector[dit->first].second[j] - records->at(i).eigenVector[j]);
					sum +=  tmp * tmp;
					dit->second = sum;	
				}
				string fileKey = indexVector[dit->first].first;

				if (angles[i].find(fileKey) == angles[i].end()) {
					vector<unsigned int> tmp;
					angles[i][fileKey] = tmp;
				}
				angles[i][fileKey].push_back(dit->first);
			}
		}

		for (unsigned int i = it->second.first, n = it->second.second + it->second.first; i < n; i++) {
			map<string, vector<unsigned int> >::iterator ait;
			for(ait = angles[i].begin(); ait != angles[i].end(); ait++) {
				double tinyDis = DBL_MAX;
				double smallDis = DBL_MAX;
				unsigned int tinyIndex = INT_MAX;
				unsigned int smallIndex = INT_MAX;
				for (int unsigned j = 0, m = ait->second.size(); j < m; j++) {
					double sum = distans[i][ait->second[j]];
					if (sum < tinyDis){
						smallDis = tinyDis;
						smallIndex = tinyIndex;
						tinyDis = sum;
						tinyIndex = ait->second[j];
					} else if (sum < smallDis) {
						smallDis = sum;
						smallIndex = ait->second[j];
					} 

				}

				if (tinyDis < 50000) {
					map<string, pair<unsigned int, unsigned int> >::iterator tmpit;
					map<string, pair<unsigned int, unsigned int> > * votePt = &(votes[voteIndex][img->GetFileName(tinyIndex)]);
					if ((tmpit = votePt->find(ait->first)) == votePt->end()) {
						pair<unsigned int, unsigned int> p(0, 0);
						(*votePt)[ait->first] = p;
					}
					if (smallIndex == INT_MAX) {
						(*votePt)[ait->first].first += 1;
					}
					if ((tinyDis / smallDis) < 0.6) {
						(*votePt)[ait->first].second += 1;
					}
				}
			}
		}
		voteIndex ++;
	}

	delete records;
	records = NULL;

	voteIndex = 0;
	for (it = queryimg->fileInfo.begin(); it != queryimg->fileInfo.end(); it ++, voteIndex++) {
		map<string, map<string, pair<unsigned int, unsigned int> > >::iterator vit;
		for (vit = votes[voteIndex].begin(); vit != votes[voteIndex].end(); vit ++) {
			map<string, pair<unsigned int, unsigned int> >::iterator sit;
			for (sit = vit->second.begin(); sit != vit->second.end(); sit ++) {
				if (sit->second.first + sit->second.second < 5) {
					sit->second.first = 0;
					sit->second.second = 0;
				}
			}
		}
	}

	FileHelper * qvidhelper = new FileHelper();
	unsigned int count = 0;
	for (it = queryimg->fileInfo.begin(); it != queryimg->fileInfo.end(); it++) {
		vector<string> strs = split(it->first, "\\");
		qvidhelper->AddFileName(strs[0], count);
		count ++;
	}
	qvidhelper->BuildPosition();
	string helpername = "qraw_vid_helper.dat";
	SaveFileHelper(qvidhelper, workDir + "\\" + helpername);
	pBreakLog->UpdateBreakInfo(S_QUERY_VID_HELPER, helpername, false);

	string name = "match_images.dat";
	string output = workDir + "\\" + name;
	ofstream ofs(output.c_str(), ios::binary);
	boost::archive::binary_oarchive oa(ofs);
	oa << votes;
	ofs.close();
	pBreakLog->UpdateBreakInfo(S_MATCH_IMG, name, false);
	pInfoLog->Log("Info: Save match result to (%s).", name.c_str());

	//4.begin:
	if (pConf->GetVal(RUN_MATCH_IMAGE) > 0) {
		string readname = "match_images.txt";
		string tmpname = workDir + "\\" + readname;
		ofstream fout(tmpname.c_str());
		voteIndex = 0;
		for (it = queryimg->fileInfo.begin(); it != queryimg->fileInfo.end(); it++) {
			fout << "===============================" << endl;
			fout << "QUERY IMAGE:"<< it->first << ":" << endl;
			map<string, map<string, pair<unsigned int, unsigned int > > >::iterator vit;
			for(vit = votes[voteIndex].begin(); vit != votes[voteIndex].end(); vit++) {
				fout << "---------------------------------" << endl;
				fout << "LIB IMAGE: " << vit->first << endl;
				fout << "----" << endl;
				map<string, pair<unsigned int, unsigned int > >::iterator sit;
				for (sit = vit->second.begin(); sit != vit->second.end(); sit++) {
					fout << "ANGLE : " << sit->first;
					fout << " | SINGLE_P SCORE: " << sit->second.first;
					fout << " | MULTI_P SCORE: " << sit->second.second << endl;
				}
			}
			voteIndex++;
		}
		fout.close();
		pBreakLog->UpdateBreakInfo(S_READABLE_RES, readname, false);
	}

	//release FileHelper
	delete img;
	img = NULL;
	delete queryimg;
	queryimg = NULL;
	pBreakLog->SetNewState(S_MATCH_VID);
	return true;
}

bool MatchVideo() {
	pInfoLog->Log("Info: Prepare data.");
	vector<map<string, map<string, pair<unsigned int, unsigned int> > > > * imgvotes = 
		new vector<map<string, map<string, pair<unsigned int, unsigned int> > > >();

	if (!LoadImgVotes(imgvotes)) {
		pInfoLog->Log("Error: Cannot load voting results from image matching.");
		return false;
	}

	FileHelper * qvidhelper = new FileHelper();
	string helpername = pBreakLog->info->fileInfo[S_QUERY_VID_HELPER][0];
	string qvname = workDir + "\\" + helpername;
	LoadFileHelper(qvidhelper, qvname);
	unsigned int vidnum = qvidhelper->fileInfo.size();
	vector<map<string, unsigned int> > vidvotes(vidnum);

	map<string, pair<unsigned int, unsigned int> >::iterator it;
	unsigned int cnt = 0;
	for (it = qvidhelper->fileInfo.begin(); it != qvidhelper->fileInfo.end(); it ++, cnt++) {
		unsigned int i = it->second.first;
		unsigned int n = it->second.first + it->second.second;
		for (; i < n; i++) {
			map<string, map<string, pair<unsigned int, unsigned int> > >::iterator it2;
			for(it2 = imgvotes->at(i).begin(); it2 != imgvotes->at(i).end(); it2 ++) {
				map<string, pair<unsigned int, unsigned int> >::iterator oit;
				for (oit = it2->second.begin(); oit != it2->second.end(); oit++) {
					if (oit->second.first != 0 || oit->second.second != 0) {
						vector<string> strs = split(it2->first, "\\");
						if (vidvotes[cnt].find(strs[0]) != vidvotes[cnt].end()) {
							vidvotes[cnt][strs[0]] += 1;
						} else {
							vidvotes[cnt][strs[0]] = 1;
						}
					}
				}
			}
		}
	}

	if (pConf->GetVal(RUN_MATCH_VIDEO) > 0) {
		string readname = "match_videos.txt";
		string tmpname = workDir + "\\" + readname;
		ofstream fout(tmpname.c_str());
		cnt = 0;
		for (it = qvidhelper->fileInfo.begin(); it != qvidhelper->fileInfo.end(); it ++, cnt ++) {
			fout << "===============================" << endl;
			fout << "QUERY IMAGE:"<< it->first << endl;
			map<string, unsigned int>::iterator vit;
			for (vit = vidvotes[cnt].begin(); vit != vidvotes[cnt].end(); vit ++) {
				fout << "---------------------------------" << endl;
				fout << "LIB VIDEO: " << vit->first << " | " << "SCORE: "<< vit->second << endl; 
			}
		}
		fout.close();
		pBreakLog->UpdateBreakInfo(S_READABLE_RES, readname, false);
	}

	delete imgvotes;
	imgvotes = NULL;
	delete qvidhelper;
	qvidhelper = NULL;
	return true;
}

template<class T>
bool SaveDataBlock(DataBlock<T> * db, const string & filename) {
	if (db->GetCount() <= 0) {
		return false;
	}
	db->FixSize();
	string output = workDir + "\\" + filename;
	ofstream ofs(output.c_str(), ios::binary);
	boost::archive::binary_oarchive oa(ofs);
	oa << *db;
	ofs.close();
	pInfoLog->Log("Info: Save data block to (%s).", filename.c_str());
	return true;
}

template<class T>
bool LoadDataBlock(DataBlock<T> * db, const string & filename) {
	assert(db != NULL);
	string output = workDir + "\\" + filename;
	ifstream ifs(output.c_str(), ios::binary);
	if (!ifs.good()) {
		pInfoLog->Log("Error: Unable to load data from (%s).", filename.c_str());
		return false;
	}
	boost::archive::binary_iarchive ia(ifs);
	ia >> *db;
	ifs.close();
	return true;
}

vector<int> ParseFileName(const string & filename) {
	unsigned int pos = filename.find('.');
	string noext = filename.substr(0, pos);
	vector<string> numsStr = split(noext, "_");
	vector<int> ret(numsStr.size() - 1);
	for (unsigned int i = 0; i < ret.size(); i++) {
		ret[i] = atoi(numsStr[i + 1].c_str());
	}
	return ret;
}

bool DeleteFiles(int key) {
	vector<string> files = pBreakLog->info->fileInfo[key];
	pBreakLog->info->fileInfo[key].clear();
	for (unsigned int i = 0; i < files.size(); i++) {
		string tmp = workDir + "\\" + files[i];
		if (remove(tmp.c_str()) == -1) {
			return false;
		}
	}
	return true;
}

void BuildQueryData(vector<Record> * records) {
	pInfoLog->Log("Info: Build query data.");
	vector<string> qrawfiles = pBreakLog->info->fileInfo[S_RAW_DATA];
	unsigned int fileNum = qrawfiles.size();
	unsigned int totalNum = 0;
	unsigned int startId = 0;

	DataBlock<Record> * qrawdb = NULL;
	for (unsigned int i = 0; i < qrawfiles.size(); i++) {
		qrawdb = new DataBlock<Record>();
		LoadDataBlock(qrawdb, qrawfiles[i]);
		unsigned int recNum = qrawdb->dataSet.size();
		startId = totalNum;
		totalNum += recNum;
		records->resize(totalNum);

		for (unsigned int j = 0; j < recNum; j++) {
			records->at(startId + j) = qrawdb->dataSet[j];
		}

		delete qrawdb;
		qrawdb = NULL;
	}
}

bool LoadSearchResults(vector<vector<vector<unsigned int> > > * results) {
	string filename = pBreakLog->info->fileInfo[S_MATCH_PNT][0];
	string input = workDir + "\\" + filename;
	ifstream ifs(input.c_str(), ios::binary);
	if (!ifs.good()) {
		return false;
	}
	boost::archive::binary_iarchive ia(ifs);
	ia >> *results;
	ifs.close();
	return true;
}

bool LoadImgVotes(vector<map<string, map<string, pair<unsigned int, unsigned int> > > > * imgvotes) {
	string filename = pBreakLog->info->fileInfo[S_MATCH_IMG][0];
	string input = workDir + "\\" + filename;
	ifstream ifs(input.c_str(), ios::binary);
	if (!ifs.good()) {
		return false;
	}
	boost::archive::binary_iarchive ia(ifs);
	ia >> *imgvotes;
	ifs.close();
	return true;
}